/*
 * Copyright (c) 2018-2023 北京华恒盛世科技有限公司
 * QUICKPOOL is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *     http://license.coscl.org.cn/MulanPSL2
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
*/

#ifndef JOB_SCHEDULER_HEADER
#define JOB_SCHEDULER_HEADER

#include "hash.h"
#include "list.h"
#include "bitset.h"
#include "channel.h"
#include "xdrbase.h"
#include "tclinfo.h"
#include "quickpool.h"
#include "jobgroup.h"
#include "lib.h"
#include "api.h"

#define DEF_PRIO             1
#define DEF_NICE             0
#define DEF_Q_SCHED_DELAY    0
#define MAX_INTERNAL_JOBID   299999999999999LL
#define DEF_PEND_EXIT       512
#define MAX_JOB_PRIORITY   INFINIT_INT

typedef enum {
    JS_CONNTIMEOUT,
    JS_RECVTIMEOUT,
} jsparams_t;
 
#define BAD_LOAD     1
#define TOO_LATE     2
#define FILE_MISSING 3
#define BAD_USER     4
#define JOB_MISSING  5
#define MISS_DEADLINE 8

#define JFLAG_FINISH_JOB_ONCE   0x0001
#define JFLAG_LASTRUN_SUCC      0x0002
#define JFLAG_DEPCOND_INVALID   0x0004
#define JFLAG_READY             0x0008
#define JFLAG_SEND_SIG          0x0010
#define JFLAG_ADM_BTOP          0x0020
#define JFLAG_READY1            0x0040
#define JFLAG_READY2            0x0080
#define JFLAG_URGENT            0x0100
#define JFLAG_URGENT_NOSTOP     0x0200
#define JFLAG_REQUEUE           0x0400
#define JFLAG_HAS_BEEN_REQUEUED 0x0800
#define JFLAG_JOB_PREEMPTED     0x1000
#define JFLAG_BORROWED_SLOTS    0x2000

#define SCHEDULE_GOT_LOAD    0x0001
#define SCHEDULE_GET_AVAIL   0x0002
#define SCHEDULE_QUE_CAND    0x0004
#define RECORD_REPLAY        0x0008
#define SCHEDULE_START       0x0010
#define SCHEDULE_RESUME_SUSP 0x0020

#define JOB_STAGE_READY 0x0001
#define JOB_STAGE_CAND  0x0002
#define JOB_STAGE_DISP  0x0004
#define JOB_STAGE_DONE  0x0008

#define DEFAULT_LISTSIZE 200

extern char *gstrClusterName;
extern struct node *gptrMasterNode;
extern char *gstrMasterName;
extern bool_t gboolJsCheckConfig;
extern int gintConfigError;
extern unsigned int gintSchedulePhase;
extern JOBID_T gintMaxJobId;

extern struct policyFairshare *gptrFairsharePolicy;
extern struct policyPreempt *gptrPreemptPolicy;
extern struct policyThreshold *gptrThresholdPolicy;
extern struct policyReservation *gptrReservationPolicy;

#define DEL_ACTION_REQUEUE   0x02

#define PUT_HIGH(word, s) (word = ((s << 16) | (word & 0x0000ffff)))
#define NUMNODE (HASH_NUM_ENTS(ghashHost)-gintLostNode-gintDummyAllNode)

/* Maximum number of messages before
 * the queue rolls over
 */
#define MAX_JOB_MSG 512

#define JOB_URGENT        0x40000000
#define JOB_URGENT_NOSTOP 0x80000000

struct resourceInstance {
    LIST_T *forw;
    LIST_T *back;
    char *resource_name;
    unsigned int flags;
    HASH_TABLE_T *node_table;
    char *org_value;
    char *value;
};

struct sharedResource {
    char *resource_name;
    unsigned int flags;
    int index;
    LIST_T *inst_list;
};

#define  JOB_PREEMPT_WAIT(s)  ((s)->job_status & JOB_STATUS_PREEMPT_WAIT)

struct readyNode {
    struct node *rnode;
    unsigned int total_slot;
    unsigned int number_free_slot;
    unsigned int none_bfslot;
    unsigned int free_none_bfslot;
    LIST_T *reservation_jobs;
    LIST_T *alloc_cores;
    LIST_T *alloc_gpus;
};

struct executeNode {
    struct node *enode;
    unsigned int number_slot;
    LIST_T *alloc_cores;
    LIST_T *alloc_gpus;
    struct jobResourceUsage resource_usage;
};

struct specifiedNode {
    struct node *snode;
    int priority;
};

#define CLEAR_REASON(v, reason) if (v == reason) v = 0;
#define SET_REASON(condition, v, reason) \
        if (condition) v = reason; else CLEAR_REASON(v, reason)

#define IS_RUN_JOB_CMD(s) (((s) & JOB_STATUS_RUN) && !((s) & JOB_STATUS_PREJOB))

struct groupReadyNode {
    unsigned int num_ready_member;
    struct readyNode *ready_members;
};

/* The jRef list is used during scheduling, its purpose
 * is to address all pending jobs so that the scheduler
 * looks at them once and once only during each
 * scheduling cycle.
 */
struct jobProxy {
    LIST_T *forw;
    LIST_T *back;
    struct job *schedule_job;
};

struct startedInfo {
    unsigned int number_alloc_slot;
    unsigned int number_exec_node;
    struct executeNode *exec_nodes;
    time_t start_time;
    pid_t ee_pid;
    pid_t ee_pgid;
    unsigned int running_time;
    float cpu_time;
    time_t resume_time;
    pid_t actcmd_pid;
    uid_t execute_uid;
    char *execute_user;
    char *execute_cwd;
    char *execute_home;
    time_t usage_update_time;
    unsigned int chkpnt_period;
};

struct job {
    LIST_T *forw;
    LIST_T *back;
    uid_t uid;
    char *user_name;
    struct user *job_user;
    JOBID_T jobid;
    unsigned int job_status;
    time_t status_time;
    unsigned int job_tag;
    int oldReason;
    int newReason;
    int subreasons;
    unsigned int numReasons;
    int *pending_reason;
    struct queue *job_queue;
    unsigned int num_spec_node;
    struct specifiedNode *spec_nodes;
    int other_spec_priority;
    unsigned int num_ready_node;
    struct readyNode *ready_nodes;
    unsigned int num_dispatch_node;
    struct readyNode *dispatch_nodes;
    unsigned int num_dispatch_slot;
    unsigned int num_slot;
    unsigned int num_free_slot;
    bool_t use_peer;
    unsigned int processed;
    int dispatch_times;
    struct startedInfo *start_info;
    struct pendSignal {
        int   notSwitched;
        int   sig;
        int   sig1;
        int   sig1Flags;
        int   sigDel;
        int   notModified;
    } pend_signal;
    char *schedule_node_type;
    struct submitRequest *newSub;
    time_t dispatch_time;
    time_t possible_start;
    time_t requeue_time;
    pid_t restart_pid;
    time_t reserve_time;
    time_t end_time;
    time_t suspend_time;
    int reque_mode;
    LIST_T *requeue_nodes;
    int exit_code;
    char *queuePreCmd;
    char *queuePostCmd;
    int start_failure_count;
    int act_signal;
    int num_depend;
    struct jShared  *shared;
    LIST_T *current_list;
    struct jgTreeNode *jgrpNode;
    int job_type;
    struct job *next_element;
    int job_priority;
    unsigned int num_ready_group;
    struct groupReadyNode *ready_groups;
    int *ready_group_bitset;
    unsigned int num_message;
    struct QuickPoolJobMsg **msgs;
    int abs_run_limit; /* absolute run limit in seconds */
    char *req_gpu_model;
    int req_gpu_exclusive;
    struct job *preempting_job;
    struct requeueExitCode *reque_code;
    int requeue_cnt;
    long max_mem;
    long max_swap;
};

struct uhgroup {
    char *group_name;
    HASH_TABLE_T *hash_member;
    HASH_TABLE_T *hash_excl_member;
    unsigned int num_subgroups;
    struct uhgroup *subgroups[MAX_GROUPS];
    unsigned int max_slots;
    char *user_shares;
};

typedef struct policyFairshare {
    int (*create_fairshare_tree)(struct queue *, unsigned int, struct uhgroup **);
    int (*create_ownership_tree)(struct queue *, unsigned int, struct uhgroup **);
    int (*update_fairshare_acct)(struct queue *, struct job *, int, int);
    int (*fairshare_session_begin)(struct queue *);
    int (*fairshare_session_end)(struct queue *);
    int (*ownership_session_begin)(struct queue *);
    int (*fairshare_choose_job)(struct queue *, struct jobProxy **);
    int (*fairshare_get_accts)(struct queue *, unsigned int *, struct share_acct ***);
} fairshare_policy_t;

typedef struct policyPreempt {
    int (*initialize_preemption)(LIST_T *);
    int (*preempt_choose_jobs)(struct queue *, LIST_T *, unsigned int);
    int (*get_preemption_slot)(struct job *, struct node *);
} preempt_policy_t;

typedef struct policyThreshold {
    int (*threshold_check)(double *, float *, struct quickpoolInfo *);
} threshold_policy_t;


typedef struct policyReservation {
    void (*reservation_check)(struct job *, time_t, bool_t *);
    void (*reserve_memory_4job)(struct job *, int);
    void (*reserve_4job)(struct job *);
    void (*setjob_starttime)(struct job *, time_t);
    int (*sum_backfill_slot)(LIST_T *);
    void (*backfill_job)(struct job *, time_t);
    void (*backfill_job_dispatchnode)(struct job *);
    LIST_T *(*new_reservation_list)(struct job *, unsigned int);
    LIST_T *(*dup_reservation)(LIST_T *);
    void (*redo_reservation)(LIST_T *);
} reservation_policy_t;

struct jShared {
    int num_reference;
    struct dptNode *root_node;
    struct submitRequest submit;
    struct requirement *job_req;
};

struct nodeAcct {
    struct node *acct_node;
    unsigned int num_running_slot;
    unsigned int num_SSUSP_slot;
    unsigned int num_USUSP_slot;
    unsigned int num_reserving_slot;
};

struct user {
    char *user_name;
    int flags;
    struct user **user_groups;
    unsigned int num_groups;
    struct uhgroup *group;
    HASH_TABLE_T *hash_node_acct;
    unsigned int max_num_slot;
    unsigned int num_job_slot;
    unsigned int num_pending_slot;
    unsigned int num_running_slot;
    unsigned int num_SSUSP_slot;
    unsigned int num_USUSP_slot;
    unsigned int num_reserving_slot;
    int    **reasonTb;
    unsigned int numSlots;
    LIST_T *jobs;
};

#define USER_GROUP_IS_ALL_USERS(UserGroup) \
     ((UserGroup)->num_subgroups == 0 && HASH_NUM_ENTS((UserGroup)->hash_member) == 0)  \

struct userAcct {
    struct user *acct_user;
    int user_id;
    unsigned int num_pending_slot;
    unsigned int num_running_slot;
    unsigned int num_SSUSP_slot;
    unsigned int num_USUSP_slot;
    unsigned int num_reserving_slot;
    int reason;
};

struct requeueExitCode {
    int type;
#define RQE_NORMAL   0
#define RQE_EXCLUDE  1
#define RQE_END     255
    int value;
};

struct queue {
    LIST_T *forw;
    LIST_T *back;
    char *queue_name;
    char *description;
    struct uhgroup *host_group;
    unsigned int total_slot;
    unsigned int num_admin;
    uid_t *admin_uids;
    char *admins;
    char *resource_requirement;
    struct requirement *queue_req;
    float *load_schedule;
    float *load_stop;
    int mig;
    int job_schedule_delay;
    int job_dispatch_interval;
    int priority;
    int nice;
    char *prejob_command;
    char *postjob_command;
    char *prepost_user;
    struct requeueExitCode *reque_code;
    char *requeue_exitcode;
    char *dispatch_window;
    LIST_T *dispatch_winlist;
    char *run_window;
    LIST_T *run_winlist;
    time_t winclose_time;
    int queue_ulimit[RESOURCE_LIMIT_NUM];
    int default_ulimit[RESOURCE_LIMIT_NUM];
    unsigned int max_num_slot;
    unsigned int job_slot_limit;
    unsigned int host_slot_limit;
    unsigned int user_slot_limit;
    unsigned int min_job_slot;
    unsigned int default_job_slot;
    HASH_TABLE_T *hash_node_acct;
    HASH_TABLE_T *hash_user_acct;
    int schedule_policy;
    int queue_status;
    unsigned int num_job_slot;
    unsigned int num_pending_slot;
    unsigned int num_running_slot;
    unsigned int num_SSUSP_slot;
    unsigned int num_USUSP_slot;
    unsigned int num_reserving_slot;
    int num_free_slot;
    int       **reasonTb;
    unsigned int       numReasons;
    unsigned int reservation_time;
    char *resume_condition;
    struct requirement *resume_cond;
    char *stop_condition;
    char *job_starter;
    int flags;
    char *suspend_command;
    char *resume_command;
    char *terminate_command;
    int quickpool_signal[QUICKPOOL_SIGNAL_NUMBER];
    struct uhgroup *user_group;
    BITSET_T *node_bitset;
    char *spec_node_names;
    struct specifiedNode *spec_nodes;
    unsigned int num_spec_node;
    int other_spec_priority;
    LIST_T *pending_jobs;
    struct queue *schedule_queue;
    char *fairshare;
    struct tree *fairshare_tree;
    uint32_t num_fairshare_slot;  /* total number of slots to fairshare */
    char *preemption;             /* preemption description */
    HASH_TABLE_T *preemptable;    /* preemtable queues hash table */
    char *ownership;              /* ownership description */
    struct tree *ownership_tree;
    uint32_t num_ownership_slot;  /* number of owned slots */
    uint32_t loan_duration;       /* resource loan max duration */
    char *chkpnt_dir;
    int chkpnt_period;
};

struct requeueNode {
    LIST_T *forw;
    LIST_T *back;
    struct node *requeue_node;
};

/* for open-source version, limit the max number of host/GPU for now*/
#define MAX_NUM_SERVER 30
#define MAX_NUM_GPU 8
 
struct node {
    char *node_name;
    unsigned int node_index;
    char *host_type;
    char *host_model;
    unsigned int numCPUs;
    float *load_schedule;
    float *load_stop;
    char *dispatch_window;
    time_t winclose_time;
    LIST_T *dispatch_winlist;
    time_t start_job_time;
    time_t poll_time;
    unsigned int comm_error_count;
    unsigned int node_status;
    HASH_TABLE_T *hash_user_acct;
    unsigned int user_slot_limit;
    int max_num_slot;
    int num_insession_slot;
    unsigned int num_job_slot;
    unsigned int num_running_slot;
    unsigned int num_SSUSP_slot;
    unsigned int num_USUSP_slot;
    unsigned int num_reserving_slot;
    int mig;
    int chkpnt_signal;
    double maxmem;
    double maxswap;
    double maxtmp;
    int *res_bitmap;
    int *load_status;
    double *real_load;
    double *load;
    int *busy_stop;
    int *busy_schedule;
    int node_tag;
    HASH_TABLE_T *inst_table;
    LIST_T *started_jobs;
    LIST_T *reserving_jobs;
    LIST_T *zombie_jobs;
    unsigned int num_GPU;
    struct hostGPUList *gpuinfo_static;
    struct hostGPULoad *gpuinfo_dynamic;
    numa_obj_t *numa;
    time_t lastJobTime;
    struct nodeAffinity *used_numa;
};

struct jmNode {
    struct jmNode *forw;
    struct jmNode *back;
    int chanfd;
    struct job *comm_job;
    struct node *comm_node;
    jm_request_t request_code;
    time_t sent_time;
    int pend_signal;
};

extern LIST_T *glistJmNode;
extern struct QuickPoolParameter *gptrParamConf;

typedef enum {
    DPT_AND             = 0,
    DPT_OR              = 1,
    DPT_NOT             = 2,
    DPT_LEFT_           = 3,
    DPT_RIGHT_          = 4,
    DPT_DONE            = 5,
    DPT_EXIT            = 6,
    DPT_STARTED         = 7,
    DPT_NAME            = 8,
    DPT_ENDED           = 9,
    DPT_COMMA           = 17,
    DPT_GT              = 18,
    DPT_GE              = 19,
    DPT_LT              = 20,
    DPT_LE              = 21,
    DPT_EQ              = 22,
    DPT_NE              = 23,
    DPT_POST_DONE       = 24,
    DPT_POST_ERR        = 25,
    DPT_TRUE            = 26,
} dptType;

#define  DP_FALSE     0
#define  DP_TRUE      1
#define  DP_INVALID  -1
#define  DP_REJECT   -2

struct dptNode {
    dptType type;
    int value;
    int updFlag;
    union {
        struct {
            struct dptNode *left;
            struct dptNode *right;
        } opr;
        struct {
            int opType;
            int exitCode;
            struct job *depend_job;
        } job;
    } dptUnion;
};

/* Information about the child switching the qp.records */
struct switch_child {
    pid_t pid;
    bool_t child_gone;
    int status;
};

#define CONF_COND 0x001

#define QUEUE_DEFAULT     0x01
#define QUEUE_NEEDPOLL    0x02
#define QUEUE_REMOTEONLY  0x04
#define QUEUE_UPDATE_USABLE 0x08

/* Various host flags */
#define HOST_UPDATE       (1 << 0)
#define HOST_NEEDPOLL     (1 << 1)
#define HOST_UPDATE_LOAD  (1 << 2)
#define HOST_JOB_RESUME   (1 << 3)
#define HOST_AUTOCONF_MXJ (1 << 5)
#define HOST_LOST_FOUND   (1 << 6)
#define HOST_AUTO_POWER_OFF   (1 << 7)
#define HOST_DUMMY_ALL    (1 << 8)

#define USER_GROUP     0x001
#define USER_DEFAULT   0x002
#define USER_INIT      0x004
#define USER_ALL       0x008

#define dptJobRec dptUnion.job.depend_job
#define dptLeft dptUnion.opr.left
#define dptRight dptUnion.opr.right

extern HASH_TABLE_T *ghashHost;
extern HASH_TABLE_T *ghashQueue;
extern HASH_TABLE_T *ghashUser;
extern HASH_TABLE_T *ghashJob;
extern HASH_TABLE_T *ghashResource;
extern HASH_TABLE_T *ghashClientNode;

extern struct uhgroup *gptrUserGroup[];
extern struct uhgroup *gptrNodeGroup[];

extern LIST_T *glistQueue;

extern LIST_T *glistFinishJob;
extern LIST_T *glistStartedJob;
extern LIST_T *glistHiddenJob;

extern struct queue *gptrLostQueue;
extern unsigned int gintLostNode;
extern unsigned int gintDummyAllNode;

extern unsigned int gintTotalSlot;
extern unsigned int gintTotalUserGroup;
extern unsigned int gintTotalHostGroup;

extern unsigned int gintNumCleanJob;
extern bool_t gboolHasPendSignal;
extern int                    qAttributes;
extern int                    **hReasonTb;
extern uid_t *guidAdminIDs;
extern char **gstrClusterAdmins;
extern unsigned int gnumClusterAdmin;
extern char *gstrMainAdmin;
extern uid_t guidMainAdminID;
extern int gintUpdateJobPriority;

extern int gintNumJmConnection;
extern struct QuickPoolHostReply *gptrNodeReply;

extern struct switch_child *swchild;

extern int calljm_get_hostload(void);
extern void calljm_get_hostinfo(int);
extern void polling_jm(time_t);

extern void check_node_window(time_t);
extern void change_hoststatus(struct node *, int);
extern int filter_hosts(struct request*, struct QuickPoolHostReply *);
extern struct node *get_node(char *, bool_t); 
extern void set_tclnode(struct Tcl_Node *, struct node *, struct node *);
extern void set_hostdata(void);
extern void add_client_node(void);
extern void add_jsnode(struct QuickPoolHostInfo *, struct node *, bool_t);

extern unsigned int get_nodes_byresreq(struct requirement *, unsigned int *, struct node **,
                                       struct node ***, struct node *, int *);
extern struct requirement *check_requirement(char *, unsigned int);
extern void update_hostload(struct job *, bool_t);
extern void js_get_resourceinfo(void);
extern struct requirement *get_merged_req(struct requirement *, struct requirement *);
extern bool_t has_span_numhost(struct requirement *);
extern struct node *get_node_bytype(char *);

extern void check_queue_window(time_t);
extern int filter_queues(struct request *, struct QuickPoolQueueReply *);
extern int manage_queue(int, char *, char *, char *, struct auth *);
extern int manage_host(int, struct node *, char *, struct auth *auth);
extern bool_t isuser_queuemember(char *, struct queue *);
extern int is_queueadmin(struct queue *, char *);
extern bool_t isnode_queuemember(struct node *, struct queue *);
extern struct hostInfo *get_hostinfo(char *);
extern void create_queue_hostset(struct queue *);
extern int get_nodeindex(void *);
extern void *get_node_byindex(int);

extern int create_job(struct submitRequest *, struct QuickPoolSubmitReply *, int, struct auth *);
extern int check_specified_hosts(unsigned int, char **, unsigned int, unsigned int *,
                                 struct specifiedNode **, int *, int, struct QuickPoolSubmitReply *);
extern int select_jobs(struct request *, struct auth *, struct job ***, struct QuickPoolJobReply *);
extern int select_array_jobs(struct request *, struct auth *, void **, struct QuickPoolJobReply *);
extern int signal_jobs(struct request *, struct auth *, struct QuickPoolJobReply *);
extern int switch_jobs(struct request *, struct auth *, struct QuickPoolJobReply *);
extern unsigned int count_nodejobs(struct node *);
extern void get_job_threshold(struct thresholds *, struct job *);
extern void free_jobinfo(struct jobInfo *);
extern void close_jmconnect_4zombiejob(struct job *);
extern int peek_job(JOBID_T, struct auth *, struct QuickPoolPeekReply *);
extern void set_nojob_message(struct QuickPoolJobReply *, unsigned int, char *,
                              char *, char *, char *, char *);
extern int migrate_jobs(struct request *, struct auth *, struct QuickPoolJobReply *);
extern int move_job(JOBID_T, int *, int, int, char *, struct auth *);
extern int force_run_job(struct request *, char *, struct auth *);
extern void abort_job(struct job *, char);
extern void change_jobstatus(struct job *, int, time_t, const char *);
extern struct job *get_job(JOBID_T);
extern void putjob_inPJL(struct job *, time_t);
extern void putjob_inZJL(struct job *);
extern struct job *takejob_offZJL(JOBID_T);
extern void movejob_2queue(struct job *, struct queue *);
extern void set_load_policy(float *, float *, struct queue *, struct node *);
extern void delete_job(JOBID_T);
extern void dump_jm_node(const char *);
extern void add_job_2hash(struct job *);
extern bool_t ready_4resume(struct job *, int *);
extern struct job *create_job_withreference(struct job *);
extern void destroy_job_withreference(struct job *);
extern void             destroySharedRef(struct jShared *);
extern struct jShared * createSharedRef (struct jShared *);
extern int get_running_time(const struct job *);

extern void update_job_priority(struct job *, int);
extern int modify_job(struct request *, struct QuickPoolSubmitReply *, struct auth *);

extern struct job *initialize_job(struct jShared *);
extern void free_job(struct job *, bool_t);
extern void add_job(struct job *);
extern struct job *get_zombiejob(JOBID_T);
extern void accumulate_job_runtime(struct job *, int, time_t);
extern void finish_signaljob(struct job *, int, int, struct request *);
extern bool_t should_lock_job(struct job *, int);
extern int signal_pending_job(struct job *, int, int, time_t);
extern void requeue_finished_job(struct job *, time_t);
extern int requeue_single_job(struct job *, int, int, char *, struct auth *);
extern int signal_single_job(struct job *, int, int, int, char *, struct auth *);
extern int signal_started_job(struct job *, int, int, int);

extern int schedule_jobs(void);
extern void schedule_preempt(void);
extern unsigned int check_user_limit(struct job *, HASH_TABLE_T *, unsigned int, int);
extern int sort_ready_nodes(struct requirement *, int, int, struct readyNode *);
extern int check_node_limit(struct node *, struct nodeAcct *, int);
extern void free_reserve_node(struct job *);
extern int fail_2startjob(struct job *, int);
extern int check_newjob_userlimit(struct job *);

extern int handle_job_submit(int, struct header *, XDR *, struct auth *);
extern int handle_job_signal(int, struct header *, XDR *, struct auth *);
extern int handle_multijobs_signal(int, struct header *, XDR *, struct auth *);
extern int handle_job_message(int, struct header *, XDR *, struct auth *);
extern int handle_job_migrate(int, struct header *, XDR *, struct auth *);
extern int handle_job_switch(int, struct header *, XDR *, struct auth *);
extern int handle_job_move(int, struct header *, XDR *, struct auth *);
extern int handle_job_modify(int, struct header *, XDR *, struct auth *);
extern int handle_job_peek(int, struct header *, XDR *, struct auth *);
extern int handle_job_forcerun(int, struct header *, XDR *, struct auth *);

extern int handle_daemon_debug(int, struct header *, XDR *, struct auth *);
extern int handle_queue_manage(int, struct header *, XDR *, struct auth *);
extern int handle_js_reconfig(int, struct header *, XDR *, struct auth *);
extern int handle_host_manage(int, struct header *, XDR *, struct auth *);

extern int handle_user_query(int, struct header *, XDR *);
extern int handle_setting_query(int, struct header *, XDR *);
extern int handle_group_query(int, struct header *, XDR *);
extern int handle_job_query(int, struct header *, XDR *, struct auth *);
extern int handle_job_message_query(int, struct header *, XDR *, struct auth *);
extern int handle_host_query(int, struct header *, XDR *);
extern int handle_queue_query(int, struct header *, XDR *);
extern int handle_resource_query(int, struct header *, XDR *);
extern int handle_job_dependence_query(int, struct header *, XDR *);

extern int handle_job_dependence_query(int, struct header *, XDR *);
extern int handle_jobgroup_create(int, struct header *, XDR *, struct auth *);
extern int handle_jobgroup_remove(int, struct header *, XDR *, struct auth *);
extern int handle_jobgroup_query(int, struct header *, XDR *);
extern int handle_jobgroup_modify(int, struct header *, XDR *, struct auth *);

extern int handle_jm_restart(int, struct header *, XDR *, struct sockaddr_in *);
extern int handle_status_update(int, struct header *, XDR *, struct sockaddr_in *);
extern void handle_newjob_reply(struct jmNode *, struct header *, XDR *);
extern void handle_probe_reply(struct jmNode *, struct header *, XDR *);
extern void handle_switchjob_reply(struct jmNode *, struct header *, XDR *);
extern void handle_signaljob_reply(struct jmNode *, struct header *, XDR *);

extern unsigned int get_queueavail_CPUs(struct queue *);
extern struct requeueExitCode *check_requeue_exitcode(char *, char *);
extern int match_exitvalue(struct requeueExitCode *, int);
extern void clean_requeue(struct queue *);
extern int set_js_debug(struct request *, char *, struct auth *);

extern int filter_groups(struct request *, void *);
extern int filter_jobgroup(struct request *, struct QuickPoolJobGroupReply *, char *);
extern char **expand_group(struct uhgroup *, char *, unsigned int *);
extern char *get_group_members(struct uhgroup *);
extern bool_t is_group_member(char *, struct uhgroup *);
extern struct uhgroup *get_hostgroup(char *);
extern struct uhgroup *get_usergroup(char *);
extern struct uhgroup *get_group(struct uhgroup **, char *, unsigned int);
extern void update_node(void);
extern void create_usergroups(void);
extern void update_job_account(struct job *, int);
extern void update_queue_account(struct job *, int, int, int, int, int, int);
extern void update_user_account(struct job *, int, int, int, int, int, int);
extern void update_reservation_account(struct job *, int);
extern void update_account_4switchjob(struct job *, struct queue *, struct queue *, int);
extern struct user *get_user(char *);
extern struct user *add_user(char *, int, const char *, int);
extern int filter_users(struct request *, struct QuickPoolUserReply *);
extern void js_die(int);
extern int is_clusteradmin(char *);
extern char *get_default_project(void);
extern int initialize_js(void);

extern struct userAcct *get_acct_4user(HASH_TABLE_T *, struct user *);
extern struct nodeAcct *get_acct_4node(HASH_TABLE_T *, struct node *);

extern void queue_prefer_nodes(struct queue *, bool_t *);
extern struct node *new_node(struct node *);

extern void log_jobcreate(struct job *);
extern void log_jobmodify(struct request *, char *);
extern void log_jobrun(struct job *, int);
extern void log_jobstarted(struct job *);
extern void log_jobstatus(struct job *);
extern void log_jobswitch(JOBID_T, char *, int, char *);
extern void log_jobmove(JOBID_T, int, int, int , char *);
extern void log_jobstarting(struct job *);
extern void log_jobmigrate(struct job *);
extern void log_jobrequeue(struct job *);
extern void log_jobclean(struct job *);
extern void log_jobmsg(struct job *, struct QuickPoolJobMsg *);
extern void log_jobsignal(struct job *, int, int, int);
extern void log_jobforcerun(struct job *, int, char *);
extern void log_jobsigact(struct job *, int, int, int, int, int);
extern void log_unfulfill(struct job *);

extern void log_jsstart(void);
extern void log_jsquit(int);
extern void log_queuestatus(struct queue *, int, uid_t, char *, char *);
extern void log_hoststatus(struct node *, int, uid_t, char *);

extern void log_newjgrp(struct jgTreeNode *);
extern void log_deljgrp(struct jgTreeNode *);
extern void log_modjgrp(struct jgTreeNode *);
extern int  init_log(void);
extern void switch_record_file(void);
extern void switch_statistics_file(void);

extern void writejs_jobfile(struct job *, struct lenData *);
extern int rmjs_jobfile(struct job *, bool_t);
extern int readjs_jobfile(struct jobInfo *, struct job *, struct lenData *);
extern int replacecmd_ofjobfile(char *, char *, char *, int);

extern unsigned int packjm_jobinfo(struct jmPackage *, struct node *);
extern bool_t calljm_startjob(struct job *, time_t);
extern int calljm_signaljob(struct job *, int, int, int);
extern int calljm_switchjob(struct job *, bool_t);
extern int probe_compute_node(struct node *);
extern int js_ready_4work(void);

extern void find_GPU_index(struct job *, struct readyNode *, unsigned int);
extern void find_numa_core(struct job *, struct readyNode *, unsigned int);

extern struct dptNode *check_dependency(char *, char *, int *, struct QuickPoolSubmitReply *);
extern int evaluate_dependency(struct dptNode *, struct job *, LIST_T *);
extern void                 freeDepCond(struct dptNode *);
extern void                 resetDepCond(struct dptNode *);
extern float get_node_instance(char *, struct node *, struct resourceInstance **);
extern void reset_resource_value(void);
extern int filter_shared_resources(struct request *, struct QuickPoolResourceReply *);
extern void update_instance_4startedjob(const struct job *);

extern void add_reservedjob_2host(struct job *);
extern void remove_reservedjob_4host(struct job *);
extern int compare_proxy_job(void *, void *);

#define LIMIT_OF_JOB(job, i) \
 (0 < (job)->shared->submit.job_ulimit[i] ? (job)->shared->submit.job_ulimit[i] : \
 (0 < (job)->job_queue->default_ulimit[i] ? (job)->job_queue->default_ulimit[i] : (job)->job_queue->queue_ulimit[i]) \
)

#define RUN_LIMIT_OF_JOB(job) LIMIT_OF_JOB(job, RESOURCE_LIMIT_RUN)
#define CPU_LIMIT_OF_JOB(job) LIMIT_OF_JOB(job, RESOURCE_LIMIT_CPU)

#define HAS_RUN_WINDOW(queue) ((NULL != (queue)->run_window) && ('\0' != (queue)->run_window[0]))

#define FREE_READY_NODES(candnode, num ) { \
    if ((NULL != candnode)) { \
        unsigned int ii; \
        for (ii=0; ii< (num); ii++) {                                 \
            gf_list_free(candnode[ii].reservation_jobs, NULL);	      \
            candnode[ii].reservation_jobs = NULL;                     \
        }                                                             \
        FREEUP((candnode)); \
        num = 0; \
    } \
}

#define FREE_GROUP_READY_NODES(job) { \
    unsigned int jj;  \
    for (jj = 0; jj < (job)->num_ready_group; jj++) { \
        struct groupReadyNode *group; \
        group = &((job)->ready_groups[jj]); \
        FREE_READY_NODES(group->ready_members, group->num_ready_member); \
    } \
    FREEUP((job)->ready_groups);\
    (job)->num_ready_group = 0; \
    FREEUP((job)->ready_group_bitset); \
}

#define USE_LOCAL      0x01
#define CHK_TCL_SYNTAX 0x02
#define PARSE_XOR      0x04

extern int add_job_group(char *, int, char *, struct auth *);
extern int remove_job_group(char *, char *, struct auth *);
extern int modify_job_group(char *, int, char *, struct auth *);
extern int check_job_group(struct job *, struct submitRequest *, struct auth *, char *);
extern int can_switch_jgrp(char *);
extern bool_t jobgroup_limit_ok(struct job *);

extern int set_execute_hosts(unsigned int, char **, struct job *, char *);

#endif /* _JS_HEADER_ */
